package mpo.serve.processer;

import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.xsoup.ElementOperator.Regex;

public class SinaProcesser implements PageProcessor {

	@SuppressWarnings("deprecation")
	private Site site = Site.me().setDomain("news.sina.com.cn")
			.addStartUrl("http://news.sina.com.cn").setCharset("gb2312");

	public Site getSite() {
		return site;
	}

	public void process(Page page) {
		String urlRegex = "http://news\\.sina\\.com\\.cn/[a-z]+/[0-9]{4}-[0-9]{2}-[0-9]{2}/[0-9]+\\.shtml";
		List<String> links = page.getHtml().links().regex(urlRegex).all();
		System.out.println(links.size());
		if (!page.getRequest().getUrl().matches(urlRegex)) {
			page.addTargetRequests(links);
		}
		if (page.getUrl().toString().matches(urlRegex)) {
			page.putField("url", page.getUrl());
			page.putField("title",
					page.getHtml().xpath("//h1[@id=artibodyTitle]/text()"));
			page.putField("keywords",
					page.getHtml().xpath("//meta[@name=keywords]/@content"));
			page.putField("description",
					page.getHtml().xpath("//meta[@name=description]/@content"));
			page.putField("content", page.getHtml().smartContent());
			page.putField("date",
					page.getHtml().xpath("//span[@id=pub_date]/text()"));
			page.putField("media",
					page.getHtml().xpath("//span[@id=media_name]/*/text()"));
			page.putField("source", "新浪");
		}
		/*
		 * for (String s : links) { System.out.println(s); }
		 */

	}

}
